/* http_fetcher.c - HTTP handling functions

    HTTP Fetcher
    Copyright (C) 2001, 2003 Lyle Hanson (lhanson@users.sourceforge.net)

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Library General Public License for more details.

    See included LICENSE file for details

 */

#include <errno.h>              /* for errno */
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>             /* assert() macro. */
#include <string.h>             /* strerror() among others (ISO C 9899) */
#ifdef WIN32
#include <winsock.h>
#define close(x) closesocket(x)
#define snprintf _snprintf
#else
#include <netdb.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#define WSACleanup() /**/
#endif
#include "http_fetcher.h"
#include "http_fetcher_internal.h"
#include "hfetcher_error_codes.h"
#include "hfetcher_error_codes_globals.h"

/* Global data */
static int timeout = DEFAULT_READ_TIMEOUT;
static char *headerBuf = NULL;
static char *host = NULL;
static char *proxy = NULL;
static char *userAgent = NULL;
static char *referer = NULL;
static int hideUserAgent = 0;
static enum hf_err_source errorSource = 0;
static enum hf_err_code hfetcher_errno = 0;
static int error_code;          /* Error codes can be overwritten by subsequent
                                 *  library calls, so this holds status codes
                                 *  for later use */
static int errorInt = 0;        /* When the error message has a %d in it,
                                 *  this variable is inserted */


    /* 
     * Actually downloads the page, registering a hit
     *  If the fileBuf passed in is NULL, the url is downloaded and then
     *  freed; otherwise the necessary space is allocated for fileBuf.
     *  The number of bytes downloaded (excluding metadata) is stored in
     *  'fileSize'.  Note that the buffer is actually fileSize + 1 in size
     *  because a NULL byte is appended to the data.
     *  Returns:
     *      the status code of the HTTP request on success, or
     *      -1 on error
     */
int hfetcher_fetch(const char *url, char **fileBuf, int *fileSize)
    {
#ifdef WIN32
    WSADATA wsaData;
#endif
    fd_set rfds;
    struct timeval tv;
    char *charIndex = NULL,
		*requestBuf = NULL,
    	*pageBuf = NULL,
		*tmp = NULL;
    int ret = -1,
		selectRet,
		statusCode,
		sock;
    size_t bytesRead = 0,
		contentLength;


    if(url == NULL)
        {
        errorSource = FETCHER_ERROR;
        hfetcher_errno = HF_NULLURL;
        return -1;
        }
    
	requestBuf = hfetcher__composeRequest(url);
    if(requestBuf == NULL)
        return -1;  /* errorSource set within */

#ifdef WIN32
    if(WSAStartup(MAKEWORD(1, 1), &wsaData) != 0)
        {
        errorInt = WSAGetLastError();
        hfetcher_errno = HF_WINSOCK_STARTUP;
        errorSource = FETCHER_ERROR;    /* No winsock error handling yet */
        return -1;
        }
#endif

    /* errorSource set in hfetcher__makeSocket */
    sock = hfetcher__makeSocket(host);
    if(sock < 0) { free(host); free(requestBuf); WSACleanup(); return -1; }

    /* host is allocated by hfetcher__composeRequest */
    free(host);

#ifdef WIN32
    if(send(sock, requestBuf, strlen(requestBuf), 0) == SOCKET_ERROR)
#else
    if(write(sock, requestBuf, strlen(requestBuf)) < 0)
#endif
        {
        error_code = errno;
        errorSource = ERRNO;
        close(sock);
        free(requestBuf);
        WSACleanup();
        return -1;
        }

	free(requestBuf);

    /* Grab enough of the response to get the metadata */
    if(headerBuf)
        free(headerBuf);    /* Free metadata from last request */
    headerBuf = hfetcher__read_header(sock);  /* errorSource set within */
    if(!headerBuf) { close(sock); WSACleanup(); return -1; }

    /* Get the return code */
    charIndex = strstr(headerBuf, "HTTP/");
    if(charIndex == NULL)
        {
        close(sock);
        errorSource = FETCHER_ERROR;
        hfetcher_errno = HF_FRETURNCODE;
        WSACleanup();
        return -1;
        }
    while(*charIndex != ' ')
        charIndex++;
    charIndex++;

    ret = sscanf(charIndex, "%d", &statusCode);
    if(ret != 1)
        {
        close(sock);
        errorSource = FETCHER_ERROR;
        hfetcher_errno = HF_CRETURNCODE;
        WSACleanup();
        return -1;
        }

    /* We'll only continue reading successful requests */
    if(statusCode < 200 || statusCode > 299)
        {
        close(sock);
        WSACleanup();
        return statusCode;  /* The user can decide what they want to do */
        }
    
    /*
     * Parse out about how big the data segment is.
     *  Note that under current HTTP standards (1.1 and prior), the
     *  Content-Length field is not guaranteed to be accurate or even present. 
     *  I just use it here so I can allocate a ballpark amount of memory.
     *
     * Note that some servers use different capitalization
     */
#ifdef __HAVE_GNU
    charIndex = strcasestr(headerBuf, "content-length:");
#else
    charIndex = strstr(headerBuf, "Content-Length:");
    if(charIndex == NULL)
        charIndex = strstr(headerBuf, "Content-length:");
#endif
    if (! charIndex ) /* No Content-Length: header found. */
        contentLength = DEFAULT_PAGE_BUF_SIZE;
    else
        {
        ret = sscanf(charIndex + strlen("content-length:"), "%u",
            &contentLength);
        if(ret < 1)
            {
            close(sock);
            errorSource = FETCHER_ERROR;
            hfetcher_errno = HF_CONTENTLEN;
            WSACleanup();
            return -1;
            }
        }
    
    /* Allocate enough memory to hold the page */
    pageBuf = malloc(contentLength);
    if(pageBuf == NULL)
        {
        error_code = errno;
        errorSource = ERRNO;
        close(sock);
        WSACleanup();
        return -1;
        }

    /* Begin reading the body of the file */
    while(ret > 0)
        {
        FD_ZERO(&rfds);
        FD_SET(sock, &rfds);
        tv.tv_sec = timeout; 
        tv.tv_usec = 0;

        if(timeout >= 0)
            selectRet = select(sock+1, &rfds, NULL, NULL, &tv);
        else        /* No timeout, can block indefinately */
            selectRet = select(sock+1, &rfds, NULL, NULL, NULL);

        if(selectRet == 0)
            {
            close(sock);
            errorSource = FETCHER_ERROR;
            hfetcher_errno = HF_DATATIMEOUT;
            errorInt = timeout;
            free(pageBuf);
            WSACleanup();
            return -1;
            }
        else if(selectRet == -1)
            {
            error_code = errno;
            errorSource = ERRNO;
            close(sock);
            free(pageBuf);
            WSACleanup();
            return -1;
            }

#ifdef WIN32
        ret = recv(sock, pageBuf + bytesRead, contentLength, 0);
        if(ret == SOCKET_ERROR)
#else
        ret = read(sock, pageBuf + bytesRead, contentLength);
        if(ret < 0)
#endif
            {
            error_code = errno;
            errorSource = ERRNO;
            close(sock);
            free(pageBuf);
            WSACleanup();
            return -1;
            }

        bytesRead += ret;

        if(ret > 0)
            {
            /* To be tolerant of inaccurate Content-Length fields, we'll
             *  allocate another read-sized chunk to make sure we have
             *  enough room.
             */
            tmp = realloc(pageBuf, bytesRead + contentLength);
            if(tmp == NULL)
                {
                error_code = errno;
                errorSource = ERRNO;
                close(sock);
                free(pageBuf);
                WSACleanup();
                return -1;
                }
			pageBuf = tmp;
            }
        }
    
    /*
     * The download buffer is too large.  Trim off the padding.
     * Note that we add one NULL byte to the end of the data, as it may not
     *  already be NULL terminated and we can't be sure what type of data it
     *  is or what the caller will do with it.
     */
    tmp = realloc(pageBuf, bytesRead + 1);
    if(tmp == NULL)
        {
        /* tmp shouldn't be null, since we're _shrinking_ the buffer,
         *  and if it DID fail, we could go on with the too-large buffer,
         *  but something would DEFINATELY be wrong, so we'll just give
         *  an error message */
        error_code = errno;
        errorSource = ERRNO;
        close(sock);
        free(pageBuf);
        WSACleanup();
        return -1;
        }
	pageBuf = tmp;
    pageBuf[bytesRead] = '\0'; /* NULL terminate the data */

    if(fileBuf == NULL) /* They just wanted us to "hit" the url */
        free(pageBuf);
    else
        *fileBuf = pageBuf;

    close(sock);
    WSACleanup();
    *fileSize = bytesRead;
    return statusCode;
    }



    /*
     * Changes/sets the proxy server.  Keeps any colon and port number that
     *  may be a part of the URL for later.
     * Returns 0 on success, -1 on error.
     */
int hfetcher_setProxy(const char *newProxy)
    {
    char *tmp, *colon;

    if(newProxy == NULL && proxy != NULL)
        {
        free(proxy);
        proxy = NULL;
        }
    else
        {
        size_t url_len;
        url_len = strlen(newProxy);

        tmp = malloc(url_len + 1);
        if(tmp == NULL) { error_code = errno; errorSource = ERRNO; return -1; }
        if(proxy != NULL) free(proxy);
        proxy = tmp;
        strncpy(proxy, newProxy, url_len);
        proxy[url_len] = 0;
        }
    
    return 0;
    }



    /*
     * Changes the User Agent.  Returns 0 on success, -1 on error. 
     */
int hfetcher_setUserAgent(const char *newAgent)
    {
    char *tmp;

    if(newAgent == NULL && userAgent != NULL)
        {
        free(userAgent);
        hideUserAgent = 1;
        userAgent = NULL;
        }
    else if(newAgent == NULL)
        hideUserAgent = 1;
    else
        {
        tmp = malloc(strlen(newAgent) + 1);
        if(tmp == NULL) { error_code = errno; errorSource = ERRNO; return -1; }
        if(userAgent != NULL) free(userAgent);
        userAgent = tmp;
        hideUserAgent = 0;
        strcpy(userAgent, newAgent);
        }

    return 0;
    }



    /*
     * Changes the Referer.  Returns 0 on success, -1 on error
     */
int hfetcher_setReferer(const char *newReferer)
    {
    char *tmp;

    if(newReferer == NULL && referer != NULL)
        {
        free(referer);
        referer = NULL;
        }
    else
        {
        tmp = malloc(strlen(newReferer) + 1);
        if(tmp == NULL) { error_code = errno; errorSource = ERRNO; return -1; }
        if(referer != NULL) free(referer);
        referer = tmp;
        strcpy(referer, newReferer);
        }
    
    return 0;
    }



    /*
     * Changes the amount of time that HTTP Fetcher will wait for data
     *  before timing out on reads.  Values <0 mean no timeout.
     */
void hfetcher_setTimeout(int seconds) { timeout = seconds; }



    /*
     * Puts the filename portion of the url into 'filename'.
     * Returns:
     *  0 on success
     *  1 when url contains no end filename (i.e., 'www.foo.com/'),
     *      and **filename should not be assumed to be valid
     *  -1 on error
     */
int hfetcher_parseFilename(const char *url, char **filename)
    {
    const char *ptr;

    if(url == NULL)
        {
        errorSource = FETCHER_ERROR;
        hfetcher_errno = HF_NULLURL;
        return -1;
        }

    ptr = strrchr(url, '/');
    if(ptr == NULL)
        /* Root level request, apparently */
        return 1;

    ptr++;
    if(*ptr == '\0') return 1;

    *filename = malloc(strlen(ptr) + 1);
    if(*filename == NULL)
        {
        error_code = errno;
        errorSource = ERRNO;
        return -1;
        }
    strcpy(*filename, ptr);

    return 0;
    }

    
    
    /*
     * Stores a pointer to the response header (metadata) of the last http
     *  request in headerData.
     * Returns:
     *  The length of the buffer, or
     *  -1 on error.
     */
int hfetcher_returnHeader(char **headerData)
    {
    if(headerData != NULL)
        {
        *headerData = headerBuf;
        return strlen(headerBuf);
        }
    else
        {
        errorSource = FETCHER_ERROR;
        hfetcher_errno = HF_NULLPTR;
        return -1;
        }
    }



    /*
     * Depending on the source of error, calls either perror() or prints
     *  an HTTP Fetcher error message to standard error
     */
void hfetcher_perror(const char *string)
    {
    if(errorSource == ERRNO)
        {
        errno = error_code;   /* Re-establish the relevant error code */
        perror(string);
        }
    else if(errorSource == H_ERRNO)
        {
#ifndef WIN32
        h_errno = error_code;
        herror(string);
#else
        /* Shouldn't happen */
        fprintf(stderr, "Internal Error.  Winsock doesn't support h_errno.\n");
#endif
        }
    else if(errorSource == FETCHER_ERROR)
        {
        const char *stringIndex;

        if(strstr(hfetcher_errlist[hfetcher_errno], "%d") == NULL)
            {
            fputs(string, stderr);
            fputs(": ", stderr);
            fputs(hfetcher_errlist[hfetcher_errno], stderr);
            fputs("\n", stderr);
            }
        else
            {
            /* The error string has a %d in it, we need to insert errorInt */
            stringIndex = hfetcher_errlist[hfetcher_errno];

            while(*stringIndex != '%')          /* Print up to the %d */
                {
                fputc(*stringIndex, stderr);
                stringIndex++;
                }
            fprintf(stderr, "%d", errorInt);    /* Print the number */
            stringIndex += 2;                   /* Skip past the %d */
            while(*stringIndex != 0)            /* Print up to the end NULL */
                {
                fputc(*stringIndex, stderr);
                stringIndex++;
                }
            fputs("\n", stderr);
            }
        }
    else
        fputs("hfetcher_perror(): http-fetcher bug!  Invalid errorSource.\n"
           "\tPlease report this to http-fetcher-devel@lists.sourceforge.net\n",
            stderr);
    }



    /* 
     * Returns a pointer to the current error description message. The
     *  message pointed to is only good until the next call to
     *  hfetcher_strerror(), so if you need to hold on to the message for
     *  a while you should make a copy of it
     */
const char *hfetcher_strerror(void)
    {
    
    if(errorSource == ERRNO)
        return strerror(errno);
    else if(errorSource == H_ERRNO)
#ifdef HAVE_HSTRERROR       /* libc5 systems don't provide hstrerror */
        return hstrerror(error_code);   /* error_code == relevant h_errno */
#else
        return hfetcher_errlist[HF_HERROR];
#endif
    else if(errorSource == FETCHER_ERROR)
        {
        if(strstr(hfetcher_errlist[hfetcher_errno], "%d") == NULL)
            return hfetcher_errlist[hfetcher_errno];
        else
            {
            /* The error string has a %d in it, we need to insert errorInt.
             *  convertedError[128] has been declared for that purpose */
            const char *originalError = hfetcher_errlist[hfetcher_errno];
            const char *stringIndex = strstr(originalError, "%d");
            convertedError[0] = 0;      /* Start off with NULL */
            assert(stringIndex > originalError);
            strncat(convertedError, originalError,      /* Copy up to %d */
                    stringIndex - originalError);
            sprintf(&convertedError[strlen(convertedError)],"%d",errorInt);
            stringIndex += 2;       /* Skip past the %d */
            strcat(convertedError, stringIndex);

            return convertedError;
            }
        }
    return hfetcher_errlist[HF_METAERROR];  /* Should NEVER happen */
    }



/************** Functions used internally by HTTP Fetcher *********************/


    /*
     * Takes a url and (depending on proxy settings) sets the appropriate
     *  host variable and composes the appropriate HTTP request.  Note that
     *  the URL (and the host variable, when set) may have a portnumber on the
     *  end.
	 * Global pointer 'host' as well as the buffer pointed to by the return
	 * 	value are allocated and must be freed by the caller.
     * Returns:
     *  a pointer to the allocated request buffer on success, or
     *  NULL on error
     */
static char *hfetcher__composeRequest(const char *url)
    {
	char *requestBuf, *tmp;
	int bufsize = DEFAULT_REQUEST_BUF_SIZE,
		tempSize = -1,
		ret;

    /*
     * Determine the host name to connect to, compose the main 'GET' line of
     *  the request.  
     */
	requestBuf = malloc(bufsize);
	if(requestBuf == NULL)
		{
        error_code = errno;
		errorSource = ERRNO;
		return NULL;
		}
    requestBuf[0] = 0;  /* NULL out the first byte */

    if(proxy != NULL)           /* Using a proxy server */
        {
        int added_protocol = 0;
        unsigned i;             /* index */
        const char * const http_string = "http://";

        /* Allocate space for the hostname */
        host = malloc(strlen(proxy) + 1);
        if(host == NULL)
            {
            error_code = errno;
            errorSource = ERRNO;
			free(requestBuf);
            return NULL;
            }
        strcpy(host, proxy);

        /*
         * Make sure there is a 'http://' in the beginning of the url to
         *  request
         */
        for(i=0; i<strlen(http_string); i++)
            {
            if(url[i] != http_string[i])    /* Isn't one, need to add it */
                {
				/* Need to prepend "http://" to the url */
				tempSize = strlen("GET http:// ") + strlen(url) +
					strlen(HTTP_VERSION) + 3; /* + 3 is for \r, \n, and NULL */
/* FIXME: In this instance (and others below), do we need to count the NULL in
   there, or does checkBufSize account for this?  Clarify.  This was fixed
   in 1.0.3, as well as in the docs.  The changes have NOT been incorporated
   into 2.0.0 yet.  Do this! */
 				if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize)
						|| snprintf(requestBuf, bufsize,
						"GET http://%s %s\r\n", url, HTTP_VERSION) < 0)
				 	{
                    error_code = errno;
					errorSource = ERRNO;
					free(host);
					free(requestBuf);
					return NULL;
					}
              	added_protocol = 1;
              	break;
                }	/* end if(url[i] != http_string[i]) */
            }	/* end for() */

        if(!added_protocol)
			{
			/* Size of the string, the +4 is for ' ', '\r', '\n', and NULL. */
			tempSize = strlen("GET ") + strlen(url) + strlen(HTTP_VERSION) + 4;
			if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize) ||
					snprintf(requestBuf, bufsize, "GET %s %s\r\n",
					url, HTTP_VERSION) < 0 )
				{
                error_code = errno;
				errorSource = ERRNO;
				free(host);
				free(requestBuf);
				return NULL;
				}
			}	/* end if(!added_protocol) */
        }	/* end if(proxy != NULL) */

    else                        /* Connecting directly */
        {
        char *url_tmp, *hostPtr, *charIndex;

        /* Copy the url passed in into a buffer we can work with, etc. */
        url_tmp = malloc(strlen(url) + 1);
        if(url_tmp == NULL)
            {
            error_code = errno;
            errorSource = ERRNO;
			free(requestBuf);
            return NULL;
            }
        strcpy(url_tmp, url);

        /* Skip past the protocol delimiter (://) */
        hostPtr = strstr(url_tmp, "://");
        if(hostPtr == NULL)
            hostPtr = url_tmp;
        else
            hostPtr += 3;

        /* Skip the hostname and find the file name */
        charIndex = strchr(hostPtr, '/');

		/* Copy the GET line into the request buffer */
		if(charIndex == NULL)
			{
   	 		/* The url has no '/' in it, assume the user is making a
		 	 * root-level request.  +3 is for '\r', '\n', and NULL */ 
			tempSize = strlen("GET / ") + strlen(HTTP_VERSION) + 3;
			if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize) ||
				snprintf(requestBuf, bufsize, "GET / %s\r\n", HTTP_VERSION) < 0)
				{
                error_code = errno;
				errorSource = ERRNO;
				free(url_tmp);
				free(requestBuf);
				return NULL;
				}
			}
		else
			{
			tempSize = strlen("GET ") + strlen(charIndex) +
				strlen(HTTP_VERSION) + 4; /* +4 is for ' ','\r','\n',and NULL */
			if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize) ||
					snprintf(requestBuf, bufsize, "GET %s %s\r\n",
					charIndex, HTTP_VERSION) < 0)
				{
                error_code = errno;
				errorSource = ERRNO;
				free(url_tmp);
				free(requestBuf);
				return NULL;
				}
			}

        /* Null out the end of the hostname if need be */
        if(charIndex != NULL)
            *charIndex = 0;

        /* Allocate space for the hostname and copy it in */
        host = malloc(strlen(hostPtr) + 1);
        if(host == NULL)
            {
            error_code = errno;
            errorSource = ERRNO;
            free(url_tmp);
			free(requestBuf);
            return NULL;
            }
        strcpy(host, hostPtr);

        free(url_tmp);
        }	/* end else() (connecting direclty) */

    /*
     * Use Host: even though HTTP 1.0 doesn't specify it.  Some servers
     *  won't play nice if we don't send Host (specifically those running
     *  virtual domains), and it shouldn't hurt anything
     */
	ret = bufsize - strlen(requestBuf);	/* Space left in buffer */
	tempSize = (int)strlen("Host: ") + (int)strlen(host) + 3;
        /* +3 for "\r\n\0" */
	if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize + 128))
		{
        error_code = errno;
		errorSource = ERRNO;
		free(host);
		free(requestBuf);
		return NULL;
		}
	strcat(requestBuf, "Host: ");
    strcat(requestBuf, host);
    strcat(requestBuf, "\r\n");


    if(referer != NULL)     /* NO default referer */
        {
		tempSize = (int)strlen("Referer: ") + (int)strlen(referer) + 3;
            /* +3 for "\r\n\0" */
		if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize))
			{
            error_code = errno;
			errorSource = ERRNO;
			free(host);
			free(requestBuf);
			return NULL;
			}
        strcat(requestBuf, "Referer: ");
        strcat(requestBuf, referer);
        strcat(requestBuf, "\r\n");
        }

    if(userAgent == NULL && !hideUserAgent)
        {
		tempSize = (int)strlen("User-Agent: ") +
			(int)strlen(DEFAULT_USER_AGENT) + (int)strlen(VERSION) + 4;
            /* +4 is for '\', '\r', '\n', and NULL */
		if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize))
			{
            error_code = errno;
			errorSource = ERRNO;
			free(host);
			free(requestBuf);
			return NULL;
			}
        strcat(requestBuf, "User-Agent: ");
        strcat(requestBuf, DEFAULT_USER_AGENT);
        strcat(requestBuf, "/");
        strcat(requestBuf, VERSION);
        strcat(requestBuf, "\r\n");
        }
    else if(!hideUserAgent)
        {
		tempSize = (int)strlen("User-Agent: ") + (int)strlen(userAgent) + 3;
            /* +3 is for '\r', '\n', and NULL */
		if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize))
			{
            error_code = errno;
			errorSource = ERRNO;
			free(host);
			free(requestBuf);
			return NULL;
			}
        strcat(requestBuf, "User-Agent: ");
        strcat(requestBuf, userAgent);
        strcat(requestBuf, "\r\n");
        }

	tempSize = (int)strlen("Connection: Close\r\n\r\n");
	if(hfetcher__checkBufSize(&requestBuf, &bufsize, tempSize))
		{
        error_code = errno;
		errorSource = ERRNO;
		free(host);
		free(requestBuf);
		return NULL;
		}
    strcat(requestBuf, "Connection: Close\r\n\r\n");

	/* Now free any excess memory allocated to the buffer */
	tmp = realloc(requestBuf, strlen(requestBuf) + 1);
	if(tmp == NULL)
		{
        error_code = errno;
		errorSource = ERRNO;
		free(host);
		free(requestBuf);
		return NULL;
		}
	requestBuf = tmp;

	/* requestBuf and host will need to be freed by caller */
    return requestBuf;
    }



    /*
     * Reads the metadata of an HTTP response.
     * Perhaps a little inefficient, as it reads 1 byte at a time, but
     *  I don't think it's that much of a loss (most headers aren't HUGE).
     *  Also, the OS will likely have it buffered anyway, so the 1 byte reads
     *  are from a buffer.
     * Returns:
     *  A pointer to the buffer containing HTTP header metadata
     *  NULL on error
     */
static char *hfetcher__read_header(int sock)
    {
    fd_set rfds;
    struct timeval tv;
    int bytesRead = 0, newlines = 0, ret, selectRet,
        headerBufSize = DEFAULT_HEADER_BUFSIZE;
    char *header, *headerPtr, *tmp;

    header = malloc(headerBufSize);
    if(header == NULL)
        {
        error_code = errno;
        errorSource = ERRNO;
	    return NULL;
        }
    header[0] = 0;
    headerPtr = header;

    while(newlines != 2)
        {
        FD_ZERO(&rfds);
        FD_SET(sock, &rfds);
        tv.tv_sec = timeout; 
        tv.tv_usec = 0;

        if(timeout >= 0)
            selectRet = select(sock+1, &rfds, NULL, NULL, &tv);
        else        /* No timeout, can block indefinately */
            selectRet = select(sock+1, &rfds, NULL, NULL, NULL);

        if(selectRet == 0)
            {
            errorSource = FETCHER_ERROR;
            hfetcher_errno = HF_HEADTIMEOUT;
            errorInt = timeout;
            free(header);
            return NULL;
            }
#ifdef WIN32
        else if(selectRet == SOCKET_ERROR)
            {
            errorInt = WSAGetLastError();
            hfetcher_errno = HF_WINSOCK_SELECT;
            errorSource = FETCHER_ERROR;    /* No winsock error handling yet */
            free(header);
            return NULL;
            }
#endif
        else if(selectRet == -1)
            {
            error_code = errno;
            errorSource = ERRNO;
            free(header);
            return NULL;
            }

        if(hfetcher__checkBufSize(&header, &headerBufSize,1))
            {
            free(header);
            return NULL;
            }
        
#ifdef WIN32
        ret = recv(sock, headerPtr, 1, 0);
        if(ret == SOCKET_ERROR)
#else
        ret = read(sock, headerPtr, 1);
        if(ret < 0)
#endif
            {
            error_code = errno;
            errorSource = ERRNO;
            free(header);
            return NULL;
            }
        bytesRead++;

        if(*headerPtr == '\r')          /* Ignore CR */
            {
            /* Basically do nothing special, just don't set newlines
             *  to 0 */
            headerPtr++;
            continue;
            }
        else if(*headerPtr == '\n')     /* LF is the separator */
            newlines++;
        else
            newlines = 0;

        headerPtr++;
        }

    headerPtr -= 3;     /* Snip the trailing LF's */
    *headerPtr = '\0';
    tmp = realloc(header, bytesRead);
    if(tmp == NULL)
        {
        error_code = errno;
        errorSource = ERRNO;
        free(header);
        return NULL;
        }
    return tmp;
    }



    /*
     * Opens a TCP socket to the host named destHost.  Note that destHost may
     *  have portnumber info on the end, otherwise the default port will be
     *  used.
     * Returns the descriptor of the opened socket.
     * 
     * Returns:
     *  socket descriptor, or
     *  -1 on error
     */
static int hfetcher__makeSocket(const char *destHost)
    {
    int sock;                                       /* Socket descriptor */
    struct sockaddr_in sa;                          /* Socket address */
    struct hostent *hp;                             /* Host entity */
    int ret;
    int port;
    char *p;

    /* Check for port number specified in URL */
    p = strchr(host, ':');
    if(p != NULL)
        {
        port = atoi(p + 1);
        *p = '\0';
        }
    else
        {
        if(proxy != NULL)
            port = DEFAULT_PROXY_PORT;
        else
            port = DEFAULT_PORT_NUMBER;
        }

    hp = gethostbyname(destHost);
    if(hp == NULL)
        {
#ifdef WIN32
        errorInt = WSAGetLastError();
        hfetcher_errno = HF_WINSOCK_GETHBN;
        errorSource = FETCHER_ERROR;
#else
        error_code = h_errno;
        errorSource = H_ERRNO;
#endif
        return -1;
        }

    /* Copy host address from hostent to (server) socket address */
    /* hp->addr should be pointing to an in_addr_t (struct in_addr). */
    /* This just asserts that hp->length is sane */
    assert(sizeof (struct in_addr) == sizeof sa.sin_addr);
    assert(sizeof sa.sin_addr == hp->h_length);

    memcpy(&sa.sin_addr, hp->h_addr, sizeof sa.sin_addr);
    sa.sin_family = hp->h_addrtype;     /* Set service sin_family to PF_INET */
    sa.sin_port = htons(port);          /* Put portnum into sockaddr */

    sock = socket(hp->h_addrtype, SOCK_STREAM, 0);
    if(sock < 0) { error_code = errno; errorSource = ERRNO; return -1; }

    ret = connect(sock, (struct sockaddr *)&sa, sizeof(sa));
    if(ret < 0) { error_code = errno; errorSource = ERRNO; return -1; }

    return sock;
    }



	/*
	 * Determines if the given NULL-terminated buffer is large enough to
	 *	concatenate the given number of characters.  If not, it attempts to
	 *	grow the buffer to fit.
	 * Returns:
	 *	0 on success, or
	 *	-1 on error (original buffer is unchanged).
	 */
static int hfetcher__checkBufSize(char **buf, int *bufsize, int more)
	{
	char *tmp;
	int roomLeft = *bufsize - (strlen(*buf) + 1);
	if(roomLeft > more)
		return 0;
	tmp = realloc(*buf, *bufsize + more + 1);
	if(tmp == NULL)
		return -1;
	*buf = tmp;
	*bufsize += more + 1;
	return 0;
	}
